it looks like there might be a difference between the high and low debt groups, the variance is decidedly higher in the high debt group.
d.both_completed %>%
ggplot(aes(x=sonarqube_issues, fill=high_debt_version)) +
geom_boxplot()
mean(d.both_completed$sonarqube_issues)
## [1] 1.659091
var(d.both_completed$sonarqube_issues)
## [1] 5.997357
d.both_completed %>%
pull(sonarqube_issues) %>%
summary()
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.000 0.000 1.000 1.659 3.000 12.000
sprintf("Variance: %.2f", var(pull(d.both_completed, sonarqube_issues)))
## [1] "Variance: 6.00"
Variable names are modeled using the negative binomial family rather than poisson since the variance is much greater than the mean.
We include high_debt_verison as a predictor in our model as this variable represent the very effect we want to measure. We also include a varying intercept for each individual to prevent the model from learning too much from single participants with extreme measurements.
We iterate over the model until we have sane priors, in this case a prior giving a 50/50 chance was chosen in both cases.
sonarqube_issues.with <- extendable_model(
base_name = "sonarqube_issues",
base_formula = "sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
base_priors = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape")
),
family = negbinomial(),
data = d.both_completed,
base_control = list(adapt_delta = 0.98)
)
prior_summary(sonarqube_issues.with(only_priors= TRUE))
prior_summary(sonarqube_issues.with(sample_prior = "only"))
pp_check(sonarqube_issues.with(sample_prior = "only"), nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
sim.size <- 1000
sim.intercept <- rnorm(sim.size, 1.5, 1)
sim.beta <- rnorm(sim.size, 0, 1)
sim.beta.diff <- exp(sim.intercept + sim.beta) - exp(sim.intercept)
sim.beta.diff.min <- sim.beta.diff
data.frame(x = sim.beta.diff.min) %>%
ggplot(aes(x)) +
geom_density() +
xlim(-10, 10) +
labs(
title = "Beta parameter prior influence",
x = "Issues difference difference (min)",
y = "Density"
)
We check the posterior distribution and can see that the model seems to have been able to fit the data well Sampling seems to also have worked well as Rhat values are close to 1 and the sampling plots look nice. #### Posterior Predictive check
pp_check(sonarqube_issues.with(), nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 29 rows containing missing values (geom_pointrange).
#### Summary
summary(sonarqube_issues.with())
## Warning: There were 1 divergent transitions after warmup. Increasing adapt_delta
## above may help. See http://mc-stan.org/misc/warnings.html#divergent-transitions-
## after-warmup
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session)
## Data: as.data.frame(data) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.49 0.35 0.02 1.28 1.00 737 1649
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 0.75 0.33 0.09 1.39 1.00 2283
## high_debt_versionfalse -0.67 0.43 -1.48 0.20 1.00 4800
## Tail_ESS
## Intercept 2787
## high_debt_versionfalse 3450
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.88 14.63 0.33 4.67 1.01 785 532
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
plot(sonarqube_issues.with(), ask = FALSE)
# default prior for monotonic predictor
edlvl_prior <- prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
loo_result <- loo(
# Benchmark model(s)
sonarqube_issues.with(),
# New model(s)
sonarqube_issues.with("modified_lines"),
sonarqube_issues.with("work_domain"),
sonarqube_issues.with("work_experience_programming.s"),
sonarqube_issues.with("work_experience_java.s"),
sonarqube_issues.with("education_field"),
sonarqube_issues.with("mo(education_level)", edlvl_prior),
sonarqube_issues.with("workplace_peer_review"),
sonarqube_issues.with("workplace_td_tracking"),
sonarqube_issues.with("workplace_pair_programming"),
sonarqube_issues.with("workplace_coding_standards"),
sonarqube_issues.with("scenario"),
sonarqube_issues.with("group")
)
loo_result[2]
## $diffs
## elpd_diff se_diff
## sonarqube_issues.with("group") 0.0 0.0
## sonarqube_issues.with("work_domain") -0.6 2.1
## sonarqube_issues.with() -1.0 1.5
## sonarqube_issues.with("workplace_coding_standards") -1.2 1.4
## sonarqube_issues.with("mo(education_level)", edlvl_prior) -1.4 1.4
## sonarqube_issues.with("scenario") -1.4 1.8
## sonarqube_issues.with("workplace_peer_review") -1.6 1.3
## sonarqube_issues.with("workplace_pair_programming") -1.6 1.4
## sonarqube_issues.with("workplace_td_tracking") -1.7 1.5
## sonarqube_issues.with("modified_lines") -1.9 1.5
## sonarqube_issues.with("work_experience_java.s") -2.3 1.4
## sonarqube_issues.with("work_experience_programming.s") -2.4 1.6
## sonarqube_issues.with("education_field") -2.5 1.7
loo_result[1]
## $loos
## $loos$`sonarqube_issues.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.8 7.8
## p_loo 6.7 1.3
## looic 155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 560
## (0.5, 0.7] (ok) 13 29.5% 112
## (0.7, 1] (bad) 3 6.8% 37
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("modified_lines")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.7 7.8
## p_loo 6.8 1.2
## looic 157.4 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 33 75.0% 727
## (0.5, 0.7] (ok) 7 15.9% 184
## (0.7, 1] (bad) 4 9.1% 105
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.4 7.6
## p_loo 8.7 1.1
## looic 154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 1137
## (0.5, 0.7] (ok) 14 31.8% 483
## (0.7, 1] (bad) 2 4.5% 1173
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("work_experience_programming.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -79.3 8.4
## p_loo 7.9 2.0
## looic 158.5 16.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 403
## (0.5, 0.7] (ok) 12 27.3% 123
## (0.7, 1] (bad) 2 4.5% 20
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("work_experience_java.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -79.1 8.2
## p_loo 7.7 1.8
## looic 158.2 16.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 33 75.0% 981
## (0.5, 0.7] (ok) 7 15.9% 185
## (0.7, 1] (bad) 4 9.1% 52
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -79.3 8.1
## p_loo 8.9 1.7
## looic 158.6 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 38
## (0.5, 0.7] (ok) 7 15.9% 138
## (0.7, 1] (bad) 7 15.9% 5
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.2 7.7
## p_loo 6.9 1.1
## looic 156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 533
## (0.5, 0.7] (ok) 11 25.0% 178
## (0.7, 1] (bad) 4 9.1% 141
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.4 8.0
## p_loo 7.3 1.4
## looic 156.8 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 617
## (0.5, 0.7] (ok) 18 40.9% 232
## (0.7, 1] (bad) 2 4.5% 78
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.5 7.9
## p_loo 7.4 1.3
## looic 157.0 15.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 605
## (0.5, 0.7] (ok) 11 25.0% 122
## (0.7, 1] (bad) 4 9.1% 162
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.4 7.9
## p_loo 7.2 1.3
## looic 156.9 15.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 459
## (0.5, 0.7] (ok) 10 22.7% 91
## (0.7, 1] (bad) 5 11.4% 146
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.0 8.1
## p_loo 8.1 1.7
## looic 156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 363
## (0.5, 0.7] (ok) 12 27.3% 272
## (0.7, 1] (bad) 5 11.4% 52
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("scenario")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.2 8.1
## p_loo 8.2 1.6
## looic 156.5 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 467
## (0.5, 0.7] (ok) 12 27.3% 113
## (0.7, 1] (bad) 4 9.1% 287
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -76.8 7.7
## p_loo 7.5 1.2
## looic 153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 32 72.7% 503
## (0.5, 0.7] (ok) 10 22.7% 233
## (0.7, 1] (bad) 2 4.5% 106
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
sonarqube_issues.with(),
sonarqube_issues.with("group"),
sonarqube_issues.with("work_domain"),
sonarqube_issues.with("workplace_coding_standards"),
sonarqube_issues.with("mo(education_level)", edlvl_prior),
#New model(s)
sonarqube_issues.with(c("work_domain","workplace_coding_standards")),
sonarqube_issues.with(c("mo(education_level)","work_domain"), edlvl_prior),
sonarqube_issues.with(c("workplace_coding_standards","mo(education_level)"), edlvl_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## sonarqube_issues.with("group") 0.0
## sonarqube_issues.with("work_domain") -0.6
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards")) -0.8
## sonarqube_issues.with() -1.0
## sonarqube_issues.with("workplace_coding_standards") -1.2
## sonarqube_issues.with("mo(education_level)", edlvl_prior) -1.4
## sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"), edlvl_prior) -1.4
## sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior) -1.5
## se_diff
## sonarqube_issues.with("group") 0.0
## sonarqube_issues.with("work_domain") 2.1
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards")) 2.0
## sonarqube_issues.with() 1.5
## sonarqube_issues.with("workplace_coding_standards") 1.4
## sonarqube_issues.with("mo(education_level)", edlvl_prior) 1.4
## sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"), edlvl_prior) 1.4
## sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior) 2.0
loo_result[1]
## $loos
## $loos$`sonarqube_issues.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.8 7.8
## p_loo 6.7 1.3
## looic 155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 560
## (0.5, 0.7] (ok) 13 29.5% 112
## (0.7, 1] (bad) 3 6.8% 37
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -76.8 7.7
## p_loo 7.5 1.2
## looic 153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 32 72.7% 503
## (0.5, 0.7] (ok) 10 22.7% 233
## (0.7, 1] (bad) 2 4.5% 106
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.4 7.6
## p_loo 8.7 1.1
## looic 154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 1137
## (0.5, 0.7] (ok) 14 31.8% 483
## (0.7, 1] (bad) 2 4.5% 1173
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.0 8.1
## p_loo 8.1 1.7
## looic 156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 363
## (0.5, 0.7] (ok) 12 27.3% 272
## (0.7, 1] (bad) 5 11.4% 52
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.2 7.7
## p_loo 6.9 1.1
## looic 156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 533
## (0.5, 0.7] (ok) 11 25.0% 178
## (0.7, 1] (bad) 4 9.1% 141
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with(c("work_domain", "workplace_coding_standards"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.6 8.0
## p_loo 9.6 1.7
## looic 155.2 16.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 494
## (0.5, 0.7] (ok) 9 20.5% 456
## (0.7, 1] (bad) 1 2.3% 131
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.3 7.7
## p_loo 9.2 1.1
## looic 156.7 15.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1281
## (0.5, 0.7] (ok) 11 25.0% 713
## (0.7, 1] (bad) 3 6.8% 203
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.2 8.0
## p_loo 8.3 1.6
## looic 156.5 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 642
## (0.5, 0.7] (ok) 13 29.5% 77
## (0.7, 1] (bad) 3 6.8% 289
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
sonarqube_issues.with(),
sonarqube_issues.with("group"),
sonarqube_issues.with("work_domain"),
sonarqube_issues.with("workplace_coding_standards"),
sonarqube_issues.with("mo(education_level)", edlvl_prior),
sonarqube_issues.with(c("mo(education_level)","work_domain"), edlvl_prior),
#New model(s)
sonarqube_issues.with(c("work_domain","workplace_coding_standards","mo(education_level)"), edlvl_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## sonarqube_issues.with("group") 0.0
## sonarqube_issues.with("work_domain") -0.6
## sonarqube_issues.with() -1.0
## sonarqube_issues.with("workplace_coding_standards") -1.2
## sonarqube_issues.with("mo(education_level)", edlvl_prior) -1.4
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards", "mo(education_level)"), edlvl_prior) -1.5
## sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior) -1.5
## se_diff
## sonarqube_issues.with("group") 0.0
## sonarqube_issues.with("work_domain") 2.1
## sonarqube_issues.with() 1.5
## sonarqube_issues.with("workplace_coding_standards") 1.4
## sonarqube_issues.with("mo(education_level)", edlvl_prior) 1.4
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards", "mo(education_level)"), edlvl_prior) 1.9
## sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior) 2.0
loo_result[1]
## $loos
## $loos$`sonarqube_issues.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.8 7.8
## p_loo 6.7 1.3
## looic 155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 560
## (0.5, 0.7] (ok) 13 29.5% 112
## (0.7, 1] (bad) 3 6.8% 37
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -76.8 7.7
## p_loo 7.5 1.2
## looic 153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 32 72.7% 503
## (0.5, 0.7] (ok) 10 22.7% 233
## (0.7, 1] (bad) 2 4.5% 106
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -77.4 7.6
## p_loo 8.7 1.1
## looic 154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 1137
## (0.5, 0.7] (ok) 14 31.8% 483
## (0.7, 1] (bad) 2 4.5% 1173
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.0 8.1
## p_loo 8.1 1.7
## looic 156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 363
## (0.5, 0.7] (ok) 12 27.3% 272
## (0.7, 1] (bad) 5 11.4% 52
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.2 7.7
## p_loo 6.9 1.1
## looic 156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 533
## (0.5, 0.7] (ok) 11 25.0% 178
## (0.7, 1] (bad) 4 9.1% 141
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with(c("mo(education_level)", "work_domain"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.3 7.7
## p_loo 9.2 1.1
## looic 156.7 15.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1281
## (0.5, 0.7] (ok) 11 25.0% 713
## (0.7, 1] (bad) 3 6.8% 203
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`sonarqube_issues.with(c("work_domain", "workplace_coding_standards", "mo(education_level)"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -78.3 8.0
## p_loo 10.7 1.6
## looic 156.6 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 32 72.7% 798
## (0.5, 0.7] (ok) 10 22.7% 258
## (0.7, 1] (bad) 2 4.5% 59
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
We pick some of our top performing models as candidates and inspect them closer.
The candidate models are named and listed in order of complexity.
We select the simplest model as a baseline.
We inspect some of our top performing models.
All models seems to have sampled nicely (rhat = 1 and fluffy plots) they also have about the same fit to the data and similar estimates for the high_debt_version beta parameter.
We select the simplest model as a baseline.
sonarqube_issues0 <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape")
),
family = negbinomial(),
data = d.both_completed,
control = list(adapt_delta = 0.97),
file = "fits/sonarqube_issues0",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues0)
## Warning: There were 1 divergent transitions after warmup. Increasing adapt_delta
## above may help. See http://mc-stan.org/misc/warnings.html#divergent-transitions-
## after-warmup
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session)
## Data: d.both_completed (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.50 0.35 0.02 1.27 1.00 806 1629
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 0.74 0.34 0.07 1.41 1.00 2488
## high_debt_versionfalse -0.65 0.44 -1.50 0.23 1.00 4294
## Tail_ESS
## Intercept 2271
## high_debt_versionfalse 2762
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.05 1.25 0.31 3.94 1.00 918 807
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues0)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.01245467 0.4727195 -1.0415672 1.0408743
## 6033d90a5af2c702367b3a96 -0.19293194 0.5278476 -1.5095095 0.7067151
## 6034fc165af2c702367b3a98 0.43213782 0.5571518 -0.3137879 1.7727020
## 603500725af2c702367b3a99 -0.14213277 0.5165855 -1.3652020 0.8344129
## 603f97625af2c702367b3a9d 0.44416480 0.5755908 -0.3398858 1.8370410
## 603fd5d95af2c702367b3a9e -0.03703627 0.4691895 -1.0553820 0.9406905
## 60409b7b5af2c702367b3a9f 0.23018380 0.4937926 -0.5627549 1.4366155
## 604b82b5a7718fbed181b336 0.37879216 0.5375917 -0.3401873 1.6490963
## 6050c1bf856f36729d2e5218 -0.15506157 0.5253208 -1.4025005 0.8335826
## 6050e1e7856f36729d2e5219 0.07096857 0.4616443 -0.8227767 1.1993138
## 6055fdc6856f36729d2e521b 0.12884237 0.4656367 -0.7191138 1.2365668
## 60589862856f36729d2e521f -0.31039486 0.6262516 -1.9367860 0.6334938
## 605afa3a856f36729d2e5222 -0.29451717 0.6351819 -1.9435010 0.5985559
## 605c8bc6856f36729d2e5223 -0.01378988 0.4576662 -0.9777489 1.0173585
## 605f3f2d856f36729d2e5224 0.11782844 0.4621454 -0.7215196 1.2256008
## 605f46c3856f36729d2e5225 -0.29320410 0.6277727 -1.9920288 0.6522350
## 60605337856f36729d2e5226 -0.29222811 0.6239834 -1.9377522 0.6389132
## 60609ae6856f36729d2e5228 -0.09936213 0.4662010 -1.1522400 0.8260732
## 6061ce91856f36729d2e522e -0.15275983 0.5156197 -1.4054788 0.7859701
## 6061f106856f36729d2e5231 -0.32119587 0.6658447 -2.0755255 0.6549789
## 6068ea9f856f36729d2e523e -0.01789478 0.4740959 -1.0164057 1.0468773
## 6075ab05856f36729d2e5247 0.06223130 0.4639792 -0.8977303 1.1145690
plot(sonarqube_issues0, ask = FALSE)
pp_check(sonarqube_issues0, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 23 rows containing missing values (geom_pointrange).
We select the best performing model with one variable.
sonarqube_issues1 <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape")
),
family = negbinomial(),
data = d.both_completed,
control = list(adapt_delta = 0.97),
file = "fits/sonarqube_issues1",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues1)
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain
## Data: d.both_completed (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.39 0.31 0.01 1.16 1.00 1368 1824
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 0.62 0.44 -0.22 1.52 1.00 3714
## high_debt_versionfalse -0.72 0.43 -1.55 0.13 1.00 5709
## work_domainAutomotive -0.42 0.59 -1.58 0.73 1.00 3865
## work_domainDevops -0.28 0.87 -1.93 1.46 1.00 6487
## work_domainEmbedded -0.68 0.89 -2.43 1.04 1.00 5973
## work_domainFinance -0.38 0.68 -1.74 0.96 1.00 4855
## work_domainMixed 0.23 0.74 -1.26 1.73 1.00 5029
## work_domainMusic 0.05 0.79 -1.49 1.58 1.00 4746
## work_domainNone -0.11 0.58 -1.24 1.03 1.00 4323
## work_domainRetail 0.03 0.75 -1.39 1.50 1.00 5659
## work_domainTelecom 0.34 0.74 -1.10 1.82 1.00 5079
## work_domainWeb 0.98 0.55 -0.16 2.03 1.00 3794
## Tail_ESS
## Intercept 3235
## high_debt_versionfalse 2721
## work_domainAutomotive 2939
## work_domainDevops 3049
## work_domainEmbedded 2966
## work_domainFinance 3035
## work_domainMixed 2747
## work_domainMusic 2921
## work_domainNone 3033
## work_domainRetail 2904
## work_domainTelecom 2782
## work_domainWeb 2848
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.43 4.29 0.35 4.78 1.00 2115 1508
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues1)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.062302805 0.4284255 -0.8038623 1.0963900
## 6033d90a5af2c702367b3a96 -0.086466842 0.4422835 -1.1860380 0.7263208
## 6034fc165af2c702367b3a98 0.139028095 0.4237590 -0.5588491 1.2445150
## 603500725af2c702367b3a99 -0.091043952 0.4396722 -1.1411955 0.7514077
## 603f97625af2c702367b3a9d 0.159626256 0.4263584 -0.5079739 1.2478253
## 603fd5d95af2c702367b3a9e 0.059692768 0.4220291 -0.7530709 1.0847568
## 60409b7b5af2c702367b3a9f 0.236372866 0.4619131 -0.4308832 1.4450993
## 604b82b5a7718fbed181b336 0.109544205 0.4064712 -0.6149302 1.1147643
## 6050c1bf856f36729d2e5218 -0.072452698 0.4679143 -1.2197940 0.8103681
## 6050e1e7856f36729d2e5219 0.034237939 0.4344018 -0.8959063 1.0283230
## 6055fdc6856f36729d2e521b 0.058905157 0.4251294 -0.7987389 1.0847843
## 60589862856f36729d2e521f -0.163521923 0.5111288 -1.5046138 0.6367056
## 605afa3a856f36729d2e5222 -0.190421112 0.5200015 -1.5426623 0.5908613
## 605c8bc6856f36729d2e5223 0.003436533 0.4470740 -0.9731989 1.0178715
## 605f3f2d856f36729d2e5224 0.192214596 0.4540024 -0.5302722 1.3626390
## 605f46c3856f36729d2e5225 -0.187699075 0.4895044 -1.4610958 0.5624858
## 60605337856f36729d2e5226 -0.188267003 0.5109586 -1.5560162 0.5776325
## 60609ae6856f36729d2e5228 -0.028377819 0.4060344 -0.9559951 0.8269868
## 6061ce91856f36729d2e522e -0.214062434 0.4856155 -1.4906555 0.5356961
## 6061f106856f36729d2e5231 -0.209020007 0.5015912 -1.5172275 0.5340190
## 6068ea9f856f36729d2e523e 0.002388252 0.4362535 -0.9695532 0.9812145
## 6075ab05856f36729d2e5247 0.053488114 0.3956305 -0.7228165 1.0120105
plot(sonarqube_issues1, ask = FALSE)
pp_check(sonarqube_issues1, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 43 rows containing missing values (geom_pointrange).
We select the best performing model with two variables.
sonarqube_issues2 <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape"),
edlvl_prior
),
family = negbinomial(),
data = d.both_completed,
control = list(adapt_delta = 0.97),
file = "fits/sonarqube_issues2",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues2)
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level)
## Data: d.both_completed (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.41 0.34 0.01 1.25 1.00 1329 2009
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 0.68 0.74 -0.73 2.17 1.00 2687
## high_debt_versionfalse -0.73 0.43 -1.56 0.14 1.00 7346
## work_domainAutomotive -0.45 0.59 -1.63 0.72 1.00 5165
## work_domainDevops -0.27 0.84 -1.90 1.41 1.00 7366
## work_domainEmbedded -0.66 0.87 -2.35 1.07 1.00 7893
## work_domainFinance -0.39 0.72 -1.78 1.04 1.00 7231
## work_domainMixed 0.25 0.77 -1.28 1.78 1.00 6312
## work_domainMusic 0.03 0.82 -1.58 1.60 1.00 7085
## work_domainNone -0.12 0.61 -1.33 1.04 1.00 4755
## work_domainRetail 0.03 0.83 -1.60 1.63 1.00 6376
## work_domainTelecom 0.34 0.75 -1.13 1.84 1.00 6205
## work_domainWeb 0.97 0.55 -0.17 2.02 1.00 4680
## moeducation_level -0.02 0.23 -0.49 0.40 1.00 2526
## Tail_ESS
## Intercept 2498
## high_debt_versionfalse 2908
## work_domainAutomotive 2580
## work_domainDevops 2932
## work_domainEmbedded 3214
## work_domainFinance 3210
## work_domainMixed 2615
## work_domainMusic 3176
## work_domainNone 3237
## work_domainRetail 3243
## work_domainTelecom 3343
## work_domainWeb 3271
## moeducation_level 2633
##
## Simplex Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1] 0.26 0.15 0.03 0.59 1.00 6137
## moeducation_level1[2] 0.24 0.14 0.03 0.56 1.00 7529
## moeducation_level1[3] 0.24 0.14 0.04 0.58 1.00 6943
## moeducation_level1[4] 0.26 0.15 0.04 0.59 1.00 6267
## Tail_ESS
## moeducation_level1[1] 2381
## moeducation_level1[2] 2759
## moeducation_level1[3] 3159
## moeducation_level1[4] 2795
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.33 2.81 0.33 4.06 1.00 2238 2251
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues2)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.06752982 0.4503658 -0.7710735 1.1634840
## 6033d90a5af2c702367b3a96 -0.08384330 0.4782614 -1.2480452 0.8284552
## 6034fc165af2c702367b3a98 0.14531607 0.4181818 -0.5299412 1.2310248
## 603500725af2c702367b3a99 -0.11004764 0.4803116 -1.2916570 0.8007176
## 603f97625af2c702367b3a9d 0.16619652 0.4532797 -0.5980245 1.3254275
## 603fd5d95af2c702367b3a9e 0.04337720 0.4539804 -0.9270605 1.0913103
## 60409b7b5af2c702367b3a9f 0.24127043 0.5084660 -0.4920085 1.5991810
## 604b82b5a7718fbed181b336 0.12347316 0.4394427 -0.6620433 1.2423290
## 6050c1bf856f36729d2e5218 -0.08069138 0.4792503 -1.2673197 0.8247837
## 6050e1e7856f36729d2e5219 0.02721607 0.4521033 -0.8826569 1.0873413
## 6055fdc6856f36729d2e521b 0.07445620 0.4739312 -0.7951090 1.2236712
## 60589862856f36729d2e521f -0.17862792 0.5366800 -1.5459415 0.7054114
## 605afa3a856f36729d2e5222 -0.20932032 0.5578669 -1.7152450 0.6161624
## 605c8bc6856f36729d2e5223 0.00331001 0.4781140 -1.0297192 1.0413638
## 605f3f2d856f36729d2e5224 0.20850641 0.4860773 -0.5275226 1.5032648
## 605f46c3856f36729d2e5225 -0.17303213 0.5255905 -1.5325325 0.6957678
## 60605337856f36729d2e5226 -0.19324510 0.5434027 -1.5788845 0.6535786
## 60609ae6856f36729d2e5228 -0.04077799 0.4381264 -1.0519090 0.8524392
## 6061ce91856f36729d2e522e -0.24174761 0.5327489 -1.6683308 0.5799865
## 6061f106856f36729d2e5231 -0.22521261 0.5681815 -1.6686407 0.6281218
## 6068ea9f856f36729d2e523e -0.01955714 0.4710601 -1.0435433 0.9995508
## 6075ab05856f36729d2e5247 0.07638554 0.4301520 -0.7741636 1.1343728
plot(sonarqube_issues2, ask = FALSE)
pp_check(sonarqube_issues2, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 47 rows containing missing values (geom_pointrange).
We select the best performing model with three variables.
sonarqube_issues3 <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level) + workplace_coding_standards",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape"),
edlvl_prior
),
family = negbinomial(),
data = d.both_completed,
control = list(adapt_delta = 0.97),
file = "fits/sonarqube_issues3",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues3)
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level) + workplace_coding_standards
## Data: d.both_completed (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.42 0.33 0.02 1.22 1.00 1232 2067
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept 0.25 0.82 -1.26 1.93 1.00
## high_debt_versionfalse -0.68 0.42 -1.51 0.19 1.00
## work_domainAutomotive -0.28 0.59 -1.47 0.86 1.00
## work_domainDevops -0.20 0.83 -1.85 1.44 1.00
## work_domainEmbedded -0.63 0.90 -2.40 1.15 1.00
## work_domainFinance -0.44 0.71 -1.84 0.98 1.00
## work_domainMixed 0.14 0.77 -1.35 1.67 1.00
## work_domainMusic 0.18 0.77 -1.31 1.65 1.00
## work_domainNone -0.28 0.61 -1.48 0.92 1.00
## work_domainRetail -0.05 0.82 -1.62 1.58 1.00
## work_domainTelecom 0.49 0.76 -1.03 1.96 1.00
## work_domainWeb 0.99 0.56 -0.10 2.05 1.00
## workplace_coding_standardsfalse 0.58 0.50 -0.41 1.55 1.00
## moeducation_level 0.00 0.22 -0.46 0.41 1.00
## Bulk_ESS Tail_ESS
## Intercept 2411 2619
## high_debt_versionfalse 5614 2897
## work_domainAutomotive 3854 3209
## work_domainDevops 4543 3308
## work_domainEmbedded 6110 2942
## work_domainFinance 5320 3088
## work_domainMixed 5350 3412
## work_domainMusic 5233 3122
## work_domainNone 3925 3247
## work_domainRetail 3938 2902
## work_domainTelecom 4614 2602
## work_domainWeb 3587 2741
## workplace_coding_standardsfalse 4376 3030
## moeducation_level 2256 2613
##
## Simplex Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1] 0.26 0.15 0.04 0.59 1.00 5207
## moeducation_level1[2] 0.25 0.14 0.03 0.58 1.00 5297
## moeducation_level1[3] 0.24 0.14 0.03 0.57 1.00 5179
## moeducation_level1[4] 0.25 0.14 0.04 0.56 1.00 5266
## Tail_ESS
## moeducation_level1[1] 2991
## moeducation_level1[2] 2669
## moeducation_level1[3] 3213
## moeducation_level1[4] 2904
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.70 4.12 0.37 6.42 1.00 1946 1939
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues3)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.028635020 0.4650921 -0.9529764 1.1047525
## 6033d90a5af2c702367b3a96 -0.056907468 0.4591343 -1.0875695 0.8908031
## 6034fc165af2c702367b3a98 0.075621564 0.4042183 -0.7049300 1.0477538
## 603500725af2c702367b3a99 -0.108523983 0.4787815 -1.3381695 0.8097602
## 603f97625af2c702367b3a9d 0.254238622 0.4909211 -0.4748888 1.4724628
## 603fd5d95af2c702367b3a9e -0.002165308 0.4542532 -1.0026400 1.0046373
## 60409b7b5af2c702367b3a9f 0.235094547 0.5000150 -0.5245863 1.5017948
## 604b82b5a7718fbed181b336 0.048235995 0.4324996 -0.7786700 1.0791168
## 6050c1bf856f36729d2e5218 -0.058117226 0.4816999 -1.1846235 0.8978553
## 6050e1e7856f36729d2e5219 0.018111423 0.4588099 -0.9306302 1.0382193
## 6055fdc6856f36729d2e521b 0.110955132 0.4670653 -0.7539327 1.2592078
## 60589862856f36729d2e521f -0.177640134 0.5437113 -1.6028797 0.7110827
## 605afa3a856f36729d2e5222 -0.185615996 0.5252093 -1.5529320 0.6811358
## 605c8bc6856f36729d2e5223 0.036596625 0.4561369 -0.9061319 1.0759475
## 605f3f2d856f36729d2e5224 0.259323011 0.5212896 -0.4982756 1.6276045
## 605f46c3856f36729d2e5225 -0.188008394 0.5261848 -1.5444333 0.6327229
## 60605337856f36729d2e5226 -0.189419569 0.5585809 -1.5710955 0.7092739
## 60609ae6856f36729d2e5228 -0.048630413 0.4509998 -1.0716695 0.8465676
## 6061ce91856f36729d2e522e -0.202652594 0.5185739 -1.5296473 0.6462789
## 6061f106856f36729d2e5231 -0.232672622 0.5650321 -1.7252327 0.6015934
## 6068ea9f856f36729d2e523e -0.040479170 0.4642363 -1.0705635 0.9746673
## 6075ab05856f36729d2e5247 0.019725881 0.4346092 -0.8879677 1.0071218
plot(sonarqube_issues3, ask = FALSE)
pp_check(sonarqube_issues2, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 42 rows containing missing values (geom_pointrange).
All candidate models look nice, none is significantly better than the others, we will proceed the simplest model: time0
We will try a few different variations of the selected candidate model.
Some participants did only complete one scenario. Those has been excluded from the initial dataset to improve sampling of the models. We do however want to use all data we can and will therefore try to fit the model with the complete dataset.
sonarqube_issues0.all <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape")
),
family = negbinomial(),
data = as.data.frame(d.completed),
control = list(adapt_delta = 0.97),
file = "fits/sonarqube_issues0.all",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues0.all)
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session)
## Data: as.data.frame(d.completed) (Number of observations: 51)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 29)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.47 0.31 0.02 1.15 1.00 988 1925
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 0.72 0.30 0.15 1.30 1.00 2383
## high_debt_versionfalse -0.74 0.38 -1.50 0.03 1.00 5268
## Tail_ESS
## Intercept 2292
## high_debt_versionfalse 2479
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.64 4.02 0.40 6.04 1.00 966 741
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues0.all)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 0.173636078 0.4841949 -0.6685669 1.3959565
## 6033d69a5af2c702367b3a95 -0.009772081 0.4233682 -0.8965674 0.8991031
## 6033d90a5af2c702367b3a96 -0.183264035 0.5042589 -1.4222800 0.6993317
## 6034fc165af2c702367b3a98 0.472625690 0.5362046 -0.2480632 1.7230565
## 603500725af2c702367b3a99 -0.148338972 0.4889821 -1.2929017 0.7316093
## 603f84f15af2c702367b3a9b 0.006647246 0.4832838 -1.0250592 1.0672565
## 603f97625af2c702367b3a9d 0.492150415 0.5538176 -0.2471494 1.7797207
## 603fd5d95af2c702367b3a9e -0.032880360 0.4549867 -1.0192375 0.9214720
## 60409b7b5af2c702367b3a9f 0.247201765 0.4597769 -0.4899644 1.3432600
## 604b82b5a7718fbed181b336 0.406730552 0.5132121 -0.3002893 1.6122102
## 604f1239a7718fbed181b33f -0.087744781 0.5016136 -1.2894850 0.9229079
## 6050c1bf856f36729d2e5218 -0.160575634 0.4855568 -1.3590852 0.7268349
## 6050e1e7856f36729d2e5219 0.089035968 0.4328635 -0.7583526 1.0952367
## 6055fdc6856f36729d2e521b 0.143497995 0.4277343 -0.6298182 1.1584860
## 60579f2a856f36729d2e521e 0.014787560 0.5081688 -1.0773255 1.1335230
## 60589862856f36729d2e521f -0.306208419 0.5719094 -1.8334665 0.5513226
## 605a30a7856f36729d2e5221 -0.163348996 0.5434790 -1.5217667 0.7985603
## 605afa3a856f36729d2e5222 -0.307536062 0.5584935 -1.6880023 0.5397037
## 605c8bc6856f36729d2e5223 -0.007779202 0.4289889 -0.9133755 0.9047837
## 605f3f2d856f36729d2e5224 0.124312672 0.4261902 -0.6619304 1.1185785
## 605f46c3856f36729d2e5225 -0.294038602 0.5699904 -1.7610530 0.5595738
## 60605337856f36729d2e5226 -0.309975971 0.5806987 -1.7427612 0.5834909
## 60609ae6856f36729d2e5228 -0.084215278 0.4444429 -1.0653602 0.8066581
## 6061ce91856f36729d2e522e -0.154423119 0.4914238 -1.3714470 0.7165640
## 6061f106856f36729d2e5231 -0.295058208 0.5740861 -1.7538530 0.5843444
## 60672faa856f36729d2e523c 0.011448144 0.4805337 -1.0249230 1.0517728
## 6068ea9f856f36729d2e523e -0.006087948 0.4387570 -0.9178091 0.9529642
## 606db69d856f36729d2e5243 -0.147488289 0.5427777 -1.4462207 0.8502030
## 6075ab05856f36729d2e5247 0.060631922 0.4297414 -0.8000794 1.0382815
plot(sonarqube_issues0.all, ask = FALSE)
pp_check(sonarqube_issues0.all, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 17 rows containing missing values (geom_pointrange).
As including all data points didn’t harm the model we will create this variant with all data points as well.
This variation includes work_experience_programming.s predictors as it can give further insight into how experience play a factor in the effect we try to measure. This is especially important as our sampling shewed towards containing less experienced developer than the population at large.
sonarqube_issues0.all.exp <- brm(
"sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_experience_programming.s",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(1.5, 1), class = "Intercept"),
prior(exponential(1), class = "sd"),
prior(gamma(0.01, 0.01), class = "shape")
),
family = negbinomial(),
data = as.data.frame(d.completed),
control = list(adapt_delta = 0.99),
file = "fits/sonarqube_issues0.all.exp",
file_refit = "on_change",
seed = 20210421
)
summary(sonarqube_issues0.all.exp)
## Family: negbinomial
## Links: mu = log; shape = identity
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_experience_programming.s
## Data: as.data.frame(d.completed) (Number of observations: 51)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 29)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.49 0.33 0.02 1.20 1.01 822 1645
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept 0.73 0.30 0.13 1.33 1.00
## high_debt_versionfalse -0.79 0.37 -1.53 -0.05 1.00
## work_experience_programming.s -0.23 0.24 -0.69 0.24 1.00
## Bulk_ESS Tail_ESS
## Intercept 2485 2801
## high_debt_versionfalse 5070 3178
## work_experience_programming.s 3670 2769
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape 1.42 3.54 0.39 5.06 1.00 1273 1349
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(sonarqube_issues0.all.exp)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 0.147840184 0.4979331 -0.7641627 1.3383668
## 6033d69a5af2c702367b3a95 -0.028514199 0.4397984 -0.9508857 0.9083819
## 6033d90a5af2c702367b3a96 -0.222893202 0.5255988 -1.5317210 0.6663731
## 6034fc165af2c702367b3a98 0.446249367 0.5394912 -0.2934604 1.7279778
## 603500725af2c702367b3a99 -0.179509010 0.5118405 -1.4171940 0.7061523
## 603f84f15af2c702367b3a9b 0.003279425 0.5000172 -1.0484572 1.0788100
## 603f97625af2c702367b3a9d 0.462495121 0.5541805 -0.2828727 1.7689162
## 603fd5d95af2c702367b3a9e -0.056143645 0.4456638 -1.0427053 0.8794621
## 60409b7b5af2c702367b3a9f 0.209564983 0.4690933 -0.5812753 1.3288603
## 604b82b5a7718fbed181b336 0.394210612 0.5191392 -0.3369162 1.6383415
## 604f1239a7718fbed181b33f -0.095165696 0.5235878 -1.3408252 0.9029356
## 6050c1bf856f36729d2e5218 -0.133827049 0.4981417 -1.3173950 0.8170554
## 6050e1e7856f36729d2e5219 0.084352597 0.4274494 -0.7306554 1.0515933
## 6055fdc6856f36729d2e521b 0.113750087 0.4441904 -0.7066539 1.1653990
## 60579f2a856f36729d2e521e -0.002984136 0.5207445 -1.1323672 1.0848517
## 60589862856f36729d2e521f -0.248061220 0.5857061 -1.7050130 0.7029869
## 605a30a7856f36729d2e5221 -0.152530991 0.5446344 -1.4926320 0.8548997
## 605afa3a856f36729d2e5222 -0.267965857 0.5857895 -1.6980712 0.6801355
## 605c8bc6856f36729d2e5223 0.005183213 0.4525239 -0.9590902 0.9754727
## 605f3f2d856f36729d2e5224 0.344340924 0.6362826 -0.6192909 1.8391673
## 605f46c3856f36729d2e5225 -0.315157687 0.6149178 -1.8639652 0.5961203
## 60605337856f36729d2e5226 -0.329762122 0.6063929 -1.8746765 0.5832530
## 60609ae6856f36729d2e5228 -0.113227855 0.4685675 -1.1995080 0.7950688
## 6061ce91856f36729d2e522e -0.178054908 0.5170940 -1.4386847 0.7666872
## 6061f106856f36729d2e5231 -0.312922164 0.5765100 -1.7629610 0.5914138
## 60672faa856f36729d2e523c 0.004497664 0.4988850 -1.0396125 1.0910313
## 6068ea9f856f36729d2e523e -0.003998804 0.4298373 -0.9263463 0.9263328
## 606db69d856f36729d2e5243 -0.126652262 0.5471305 -1.4410468 0.9324305
## 6075ab05856f36729d2e5247 0.036304427 0.4425538 -0.8493844 1.0117328
plot(sonarqube_issues0.all.exp, ask = FALSE)
pp_check(sonarqube_issues0.all.exp, nsamples = 200, type = "bars") + xlim(-1, 15)
## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.
## Warning: Removed 24 rows containing missing values (geom_pointrange).
This means that our final model, with all data points and experience predictors, is sonarqube_issues.all.exp
To begin interpreting the model we look at how it’s parameters were estimated. As our research is focused on how the outcome of the model is effected we will mainly analyze the \(\beta\) parameters.
mcmc_areas(sonarqube_issues0.all.exp, pars = c("b_high_debt_versionfalse", "b_work_experience_programming.s"), prob = 0.95) + scale_y_discrete() +
scale_y_discrete(labels=c("High debt version: false", "Professional programming experience")) +
ggtitle("Beta parameters densities in time model", subtitle = "Shaded region marks 95% of the density. Line marks the median")
We start by extracting posterior samples
scale_programming_experience <- function(x) {
(x - mean(d.completed$work_experience_programming))/ sd(d.completed$work_experience_programming)
}
unscale_programming_experience <- function(x) {
x * sd(d.completed$work_experience_programming) + mean(d.completed$work_experience_programming)
}
post_settings <- expand.grid(
high_debt_version = c("false", "true"),
session = NA,
work_experience_programming.s = sapply(c(0, 3, 10, 25, 40), scale_programming_experience)
)
post <- posterior_predict(sonarqube_issues0.all.exp, newdata = post_settings) %>%
melt(value.name = "estimate", varnames = c("sample_number", "settings_id")) %>%
left_join(
rowid_to_column(post_settings, var= "settings_id"),
by = "settings_id"
) %>%
mutate(work_experience_programming = unscale_programming_experience(work_experience_programming.s)) %>%
select(
estimate,
high_debt_version,
work_experience_programming
)%>%
mutate(estimate = estimate)
ggplot(post %>% filter(work_experience_programming == 10), aes(x=estimate, fill = high_debt_version)) +
geom_bar(alpha = 0.5 , position = "dodge2") +
scale_fill_manual(
name = "Debt version",
labels = c("Low debt", "High debt"),
values = c("lightblue", "darkblue")
) +
facet_grid(rows = vars(work_experience_programming)) +
labs(
title = "SonarQube issues generated",
x = "Rate",
y = "Incidence rate"
) +
xlim(-1, 10) +
scale_y_continuous(limits = NULL, breaks = c(500,1000,1500,2000,2500), labels = c("10%","20%","30%","40%","50%"))
## Warning: Removed 132 rows containing non-finite values (stat_count).
## Warning: Removed 1 rows containing missing values (geom_bar).
We can also plot the difference between good variable names for the high debt version and the low debt version.
ggplot(post, aes(x=estimate, fill = high_debt_version)) +
geom_bar(alpha = 0.5 , position = "dodge2") +
scale_fill_manual(
name = "Debt version",
labels = c("Low debt", "High debt"),
values = c("lightblue", "darkblue")
) +
facet_grid(rows = vars(work_experience_programming)) +
labs(
title = "SonarQube issues generated",
x = "Rate",
y = "Incidence rate"
) +
xlim(-1, 10) +
scale_y_continuous(limits = NULL, breaks = c(500,1000,1500,2000,2500), labels = c("10%","20%","30%","40%","50%"))
## Warning: Removed 772 rows containing non-finite values (stat_count).
## Warning: Removed 5 rows containing missing values (geom_bar).
post.diff <- post %>% filter(high_debt_version == "true")
post.diff$estimate = post.diff$estimate - filter(post, high_debt_version == "false")$estimate
ggplot(post.diff, aes(x=estimate)) +
geom_bar(quantile_lines = TRUE, quantile_fun = hdi, vline_linetype = 2) +
xlim(-10, 10) +
facet_grid(rows = vars(work_experience_programming)) +
labs(
title = "SonarQube issues generater / years of programming experience",
subtitle = "Difference as: high debt issues - low debt issues",
x = "Time (min)",
y = "Density"
)
mean(post.diff)
## Warning in mean.default(post.diff): argument is not numeric or logical:
## returning NA
## [1] NA